import requests
from lxml import etree
import json

class BtcSpider(object):
    def __init__(self):
        self.base_url = 'https://www.chainnode.com/forum/61-'
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"
        }
        self.data_list = []
    # 1.发送请求
    def get_response(self, url):
        response = requests.get(url,headers=self.headers)
        # 需要看一下抓取网页的编码格式  解码的时候需要使用对应的编码格式
        data = response.content.decode('utf-8')
        return data

    # 2.解析数据
    def pase_data(self, data):
        # 使用xpath解析当前页面所有的title和url保存
        # 2.1 转类型
        x_data = etree.HTML(data)
        # 2.2 根据xpath路径开始解析  跨节点取所有a标签
        # link-dark-major font-bold bbt-block
        # link-dark-major font-bold bbt-block
        title_list = x_data.xpath('//a[@class="link-dark-major font-bold bbt-block"]/text()')
        # print(title_list)
        # 取a标签 里面的  url
        url_list = x_data.xpath('//a[@class="link-dark-major font-bold bbt-block"]/@href')
        # 添加域名
        n = len(url_list)
        for i in range(0, n):
            url_list[i] = "https://www.chainnode.com" + url_list[i]
        # print(url_list)

        # 组合  一条评论对应一个链接
        # 枚举遍历
        for index, title in enumerate(title_list):
            comment = {}
            comment['name'] = title
            comment['url'] = url_list[index]
            self.data_list.append(comment)

        # print(self.data_list)
        print(len(self.data_list))

    # 3.保存数据
    def save_data(self):
        # 将解析之后的list 列表数据转换为json串
        data_str = json.dumps(self.data_list, ensure_ascii=False)
        with open('02_bic.json', 'w', encoding='utf-8') as f:
            f.write(data_str)
    # 4.启动
    def run(self):
        for i in range(1, 5):
            # 4.1 拼接完整的url
            url = self.base_url + str(i) + ".html"
            print(url)
            # 4.2 发送请求
            data = self.get_response(url)
            # 4.3 做解析
            self.pase_data(data)
        # 4.4 做保存
        self.save_data()


BtcSpider().run()